package indi.lddsdu.spider.conponent;

import indi.lddsdu.spider.bean.RuntimeParameter;
import indi.lddsdu.spider.enums.ContentType;
import indi.lddsdu.spider.utils.ProcessUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;

/**
 * Created by jack on 18/6/4.
 */

@Component
public class ViewProcessor implements PageProcessor {
    private final String picTitle = "图片";
    private final String defaultTitle = "页面";
    private Site site = new Site().setRetryTimes(3).setRetrySleepTime(1000).setTimeOut(3000);
    private Logger logger = LoggerFactory.getLogger(getClass());
    private ProcessUtil processUtil;

    @Autowired
    public ViewProcessor(ProcessUtil processUtil) {
        this.processUtil = processUtil;
    }


    /**
     * 处理获取到的页面，将其存储进数据库中
     * @param page
     */
    public void process(Page page) {
        String pageRequestUrl = page.getRequest().getUrl();
        //判断当前页面是否需要抽取链接
        if (pageRequestUrl.contains(RuntimeParameter.domain)){
            List<String> links = page.getHtml().links().all();
            links = linksFilter(links);
            for( String link : links){
                if (!processUtil.alreadyInDatabase(link)){
                    page.addTargetRequest(link);
                }
            }
        }else{
            logger.info(String.format(""));
        }
        //获取当前页面的内容
        if (pageRequestUrl.endsWith(".jpg") || pageRequestUrl.endsWith(".png")){
            page.putField("type", ContentType.IMAGE);
            page.putField("content",page.getBytes());
        }else{
            //当前界面非图片，当做纯文本来处理
            page.putField("type",ContentType.TEXT);
            Html html = page.getHtml();
            String title = defaultTitle;
            try {
                title = Html2Text(html.css("title").get());
            }catch (Exception e){
                logger.warn("处理title时出错，出错页面" + page.getUrl() + ", 使用默认的title, e : " + e);
            }
            page.putField("title", title);
            String text = Html2Text(html.getFirstSourceText());
            page.putField("text", text);
        }
        page.putField("url", pageRequestUrl);
    }


    /**
     * 将html文件中的标签过滤出去
     * reference : https://blog.csdn.net/qiaqia609/article/details/11018407
     * 从上面那个url抄来的，楼主好人
     * @param inputString
     * @return
     */
    public static String Html2Text(String inputString) {
        String htmlStr = inputString; //含html标签的字符串
        String textStr ="";
        java.util.regex.Pattern p_script;
        java.util.regex.Matcher m_script;
        java.util.regex.Pattern p_style;
        java.util.regex.Matcher m_style;
        java.util.regex.Pattern p_html;
        java.util.regex.Matcher m_html;

        try {
            String regEx_script = "<[\\s]*?script[^>]*?>[\\s\\S]*?<[\\s]*?\\/[\\s]*?script[\\s]*?>"; //定义script的正则表达式{或<script[^>]*?>[\\s\\S]*?<\\/script> }
            String regEx_style = "<[\\s]*?style[^>]*?>[\\s\\S]*?<[\\s]*?\\/[\\s]*?style[\\s]*?>"; //定义style的正则表达式{或<style[^>]*?>[\\s\\S]*?<\\/style> }
            String regEx_html = "<[^>]+>"; //定义HTML标签的正则表达式

            p_script = Pattern.compile(regEx_script,Pattern.CASE_INSENSITIVE);
            m_script = p_script.matcher(htmlStr);
            htmlStr = m_script.replaceAll(""); //过滤script标签

            p_style = Pattern.compile(regEx_style,Pattern.CASE_INSENSITIVE);
            m_style = p_style.matcher(htmlStr);
            htmlStr = m_style.replaceAll(""); //过滤style标签

            p_html = Pattern.compile(regEx_html,Pattern.CASE_INSENSITIVE);
            m_html = p_html.matcher(htmlStr);
            htmlStr = m_html.replaceAll(""); //过滤html标签

            textStr = htmlStr;

        }catch(Exception e) {
            System.err.println("Html2Text: " + e.getMessage());
        }

        return textStr;//返回文本字符串
    }


    /**
     * 删除明显的url不正确的链接,以及只添加http://www.view.sdu.cn站内的链接
     * @param links
     * @return
     */
    public List<String> linksFilter(List<String> links){
        //TODO 扩展添加增量爬取的功能
        List<String> newLinks = new ArrayList<String>();
        for( String link : links){
            if(link.trim().equals("") || link.startsWith("mailto")){
                continue;
            }
            if (link.startsWith(RuntimeParameter.domain)) {
                newLinks.add(link);
            }
        }
        return newLinks;
    }


    public Site getSite() {
        return site;
    }

}
